<?php

require_once "Zend/Tokenizer/Interface.php";

/**
 * @category   Zend
 * @package    Zend_Tokenizer
 * @copyright  Salomon BRYS
 * @license    http://framework.zend.com/license/new-bsd     New BSD License
 */
class Zend_Tokenizer_Lexer implements Zend_Tokenizer_Interface
{
	/**
	 * @var array
	 */
	protected $_rules = array();

	/**
	 * @var Zend_Tokenizer_Interface
	 */
	protected $_producer;

	/**
	 * @var string
	 */
	protected $_data;

	/**
	 * @var array
	 */
	protected $_lexemes = array();

	public function __construct(Zend_Tokenizer_Interface $producer, array $rules = array())
	{
		if (!$producer->isStringToken()) {
			require_once 'Zend/Exception.php';
			throw new Zend_Exception("Zend_Tokenizer_Lexer needs a string token producer.");
		}
		$this->_producer = $producer;
		$this->addRules($rules);
	}

	public function addRules(array $rules)
	{
		foreach ($rules as &$regexp) {
			if ($regexp[1] != '^') {
				$regexp = trim($regexp);
				$regexp = $regexp[0] . '^' . substr($regexp, 1);
			}
		}
		$this->_rules = array_merge($this->_rules, $rules);
	}

	public function emptyRules(array $rules)
	{
		$this->_rules = array();
	}

	/**
	 * Add the buffered data not tokenized yet.
	 * Used to transfer data when tokenizer is changing on the fly.
	 *
	 * @return string
	 */
	public function getRawData()
	{
		return  . $this->_data . $this->_producer->getRawData();
	}

	/**
	 * return wether there's at least one token ready.
	 *
	 * @return bool
	 */
	public function hasToken()
	{
		$this->_fillDataAndLexeme();
		return !empty($this->_lexemes);
	}

	/**
	 * Return the next token.
	 *
	 * @return mixed
	 */
	public function getToken()
	{
		$this->_fillDataAndLexeme();
		if (empty($this->_lexemes)) {
			return null;
		}
		return array_shift($this->_lexemes);
	}

	/**
	 * Fill the _data propriety with producer's string tokens.
	 */
	protected function _fillDataAndLexeme()
	{
		while ($this->_producer->hasToken()) {
			$this->_data .= $this->_producer->getToken();
		}

		$cont = true;
		while ($cont) {
			$cont = false;
			foreach ($this->_rules as $name => $regexp) {
				$matches = array();
				if (preg_match($regexp, $this->_data, $matches)) {
					if (strlen($matches[0]) != strlen($this->_data) || $this->_producer->isEndOfStream()) {
						$this->_lexemes[] = array('type' => $name, 'value' => $matches[0]);
						$this->_data = substr($this->_data, strlen($matches[0]));
						$cont = true;
					}
					break ;
				}
			}
		}
	}

	/**
	 * Return wether it is the of the stream.
	 *
	 * @return bool
	 */
	public function isEndOfStream()
	{
		return $this->_producer->isEndOfStream();
	}

	/**
	 * Get the 'first' tokenizer. The producer.
	 * If a decorator, returns the decorated tokenizer.
	 * If a producer, returns itself.
	 *
	 * @return Zend_Tokenizer_Interface
	 */
	public function getProducer()
	{
		return $this->_producer;
	}

}
